{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "ok\n"
    }
   ],
   "source": [
    "#%%\n",
    "print(\"ok\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "1105\n374\n"
    }
   ],
   "source": [
    "allNormalText = open(r\"./data/data-normal.txt\", 'r', encoding='utf-8').read()\n",
    "allNormalTextArray = allNormalText.split('\\n')\n",
    "print(len(allNormalTextArray))\n",
    "\n",
    "allAdText = open(r\"./data/data-pssisterad.txt\", 'r', encoding='utf-8').read()\n",
    "allAdTextArray = allAdText.split('\\n')\n",
    "print(len(allAdTextArray))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "Building prefix dict from the default dictionary ...\nDumping model to file cache C:\\Users\\Kenvix\\AppData\\Local\\Temp\\jieba.cache\nLoading model cost 1.048 seconds.\nPrefix dict has been built successfully.\n1479\n"
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "[['没有', '上次', '学长', '学姐', '教', '几节课'],\n ['学姐', '交', '一下'],\n ['认识', '学姐', '保研'],\n ['党明', '学姐', '真的', '强'],\n ['想不到', '没有', '党明', '学姐', '做', '不到'],\n ['党明', '学姐'],\n ['党明', '学姐', '献上'],\n ['学姐'],\n ['我见', '证明', '党明', '学姐', '非常', '漂亮', '女孩子'],\n ['开机', '慢点', '还好']]"
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "source": [
    "from gensim.models import Word2Vec\n",
    "from model import AdPredictor\n",
    "adp = AdPredictor()\n",
    "splitNormalWords = [list(adp.splitWords(ad, False)) for ad in allNormalTextArray+allAdTextArray]\n",
    "print(len(splitNormalWords))\n",
    "splitNormalWords[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "None\n[[(0, 1), (1, 1), (2, 1), (3, 1), (4, 1), (5, 1)], [(2, 1), (6, 1), (7, 1)], [(2, 1), (8, 1), (9, 1)], [(2, 1), (10, 1), (11, 1), (12, 1)], [(2, 1), (5, 1), (10, 1), (13, 1), (14, 1), (15, 1)], [(2, 1), (10, 1)], [(2, 1), (10, 1), (16, 1)], [(2, 1)], [(2, 1), (10, 1), (17, 1), (18, 1), (19, 1), (20, 1), (21, 1)], [(22, 1), (23, 1), (24, 1)]]\n"
    }
   ],
   "source": [
    "from gensim import corpora\n",
    "# 赋给语料库中每个词(不重复的词)一个整数id\n",
    "dictionary = corpora.Dictionary(splitNormalWords)\n",
    "new_corpus = [dictionary.doc2bow(text) for text in splitNormalWords]\n",
    "print(dictionary.get(\"学姐\"))\n",
    "print(new_corpus[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "[[(0, 0.43486697887608194), (1, 0.5119171361324011), (2, 0.31321233740410154), (3, 0.41469066267165816), (4, 0.46330389940202965), (5, 0.2532009857936268)], [(2, 0.5465089583938454), (6, 0.4653393010174748), (7, 0.6962666826179764)], [(2, 0.49641058193340654), (8, 0.6572436925228606), (9, 0.567104278579744)], [(2, 0.4157386673474663), (10, 0.5296619173467123), (11, 0.6149609153670653), (12, 0.4104176974201508)], [(2, 0.35642683093967287), (5, 0.2881356006126524), (10, 0.45409708910125546), (13, 0.37192254000856334), (14, 0.3256468748375844), (15, 0.5825473033649264)]]\n2\n153\n上次\n"
    }
   ],
   "source": [
    "from gensim import models\n",
    "tfidf = models.TfidfModel(new_corpus)\n",
    "tfidf.save(\"tfidf.gsm\")\n",
    "\n",
    "# 使用这个训练好的模型得到单词的tfidf值\n",
    "tfidf_vec = []\n",
    "tfidf_vocabs = []\n",
    "for i in range(len(splitNormalWords)):\n",
    "    string_bow = dictionary.doc2bow(splitNormalWords[i])\n",
    "    string_tfidf = tfidf[string_bow]\n",
    "    tfidf_vec.append(string_tfidf)\n",
    "\n",
    "print(tfidf_vec[0:5])\n",
    "tfidf_vocabs_word_to_id = dictionary.token2id\n",
    "print(tfidf_vocabs_word_to_id.get(\"学姐\"))\n",
    "print(tfidf_vocabs_word_to_id.get(\"大佬\"))\n",
    "# print(tfidf_vocabs)\n",
    "tfidf_vocabs = {value:key for (key, value) in tfidf_vocabs_word_to_id.items()}\n",
    "print(tfidf_vocabs[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "1479\n"
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "array([[ 0.03726877, -0.0260108 ,  0.00364225, -0.00761493,  0.0261371 ,\n         0.02732867,  0.05900647,  0.0104771 , -0.03713089, -0.00172936],\n       [ 0.05519021, -0.03163468,  0.02293371,  0.00446185,  0.04841976,\n         0.00746298,  0.04423537, -0.01965555, -0.02958047,  0.0137738 ],\n       [ 0.03354345, -0.01383114, -0.01749373,  0.00576716, -0.00325754,\n         0.0032355 ,  0.04443578, -0.01131739, -0.0082487 ,  0.02331621],\n       [ 0.05156073,  0.00720141,  0.01553649, -0.01488   ,  0.04254658,\n         0.01541946,  0.05828768, -0.02861129, -0.04046905,  0.01433513],\n       [ 0.0414235 , -0.01066572,  0.00533527, -0.02395561,  0.04568579,\n         0.00993827,  0.05862336, -0.01847712, -0.04673702,  0.02249537],\n       [ 0.03303693,  0.01511562, -0.00361795, -0.027462  ,  0.02465398,\n         0.02000896,  0.05029019, -0.02054171, -0.03193081,  0.02003735],\n       [ 0.03303693,  0.01511563, -0.00361795, -0.027462  ,  0.02465398,\n         0.02000896,  0.05029019, -0.02054171, -0.03193081,  0.02003735],\n       [ 0.06617122, -0.01597482,  0.01829098, -0.0078691 ,  0.04856545,\n         0.00843941,  0.05450871,  0.01873803, -0.07134406,  0.04246678],\n       [ 0.02924464, -0.00462915, -0.00646438, -0.02488411,  0.00983   ,\n         0.01029698,  0.03802389, -0.00892737, -0.04156737,  0.02693516],\n       [ 0.0205241 , -0.067721  ,  0.01955847, -0.03661255, -0.01001304,\n         0.01028287,  0.03337859,  0.00900338, -0.0563796 ,  0.05113092],\n       [ 0.02061952, -0.06362686,  0.03869297, -0.04093252,  0.03748391,\n         0.01056947,  0.04970839, -0.01326662, -0.07499063,  0.03201511],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [ 0.07113041, -0.05779506,  0.03909116,  0.01664225,  0.08603613,\n         0.0129879 ,  0.04635942, -0.07804433,  0.03335983,  0.04454708],\n       [ 0.07113041, -0.05779506,  0.03909116,  0.01664225,  0.08603613,\n         0.01298789,  0.04635942, -0.07804432,  0.03335983,  0.04454708],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [ 0.00853717, -0.03805243,  0.00970995,  0.0213368 ,  0.02429332,\n        -0.01392098,  0.01894752,  0.00901922, -0.02115065,  0.00327003],\n       [ 0.01183973, -0.04745114, -0.01618965,  0.00339759, -0.01174291,\n         0.00504119,  0.01784882, -0.04434686, -0.03423206, -0.01691118],\n       [ 0.08888888, -0.03542721,  0.0577636 , -0.01855924,  0.09111634,\n        -0.02007198,  0.07898062, -0.03313564, -0.07436296,  0.00658349],\n       [-0.01413145, -0.00591696,  0.01551648,  0.00821014, -0.00211295,\n        -0.00301566,  0.0262262 , -0.00323604, -0.00858302,  0.01300213],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [-0.01240596, -0.00517487,  0.02039101, -0.0223156 , -0.00432781,\n         0.01305339,  0.04801728, -0.01724123, -0.01437117,  0.03366797],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [-0.00930463,  0.01191189,  0.04734699, -0.02563091, -0.02344004,\n         0.03209774,  0.03111189, -0.00334412,  0.02074025,  0.03828485],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [ 0.0951254 , -0.06806257,  0.01757841,  0.02506442,  0.04886477,\n         0.02493556,  0.00814362, -0.0501379 , -0.02636783,  0.06894183],\n       [ 0.01165463, -0.03563712,  0.02305223, -0.01200991,  0.01021166,\n         0.02734868,  0.04196411, -0.04649805, -0.03952346,  0.0477515 ],\n       [ 0.03875959, -0.00412298,  0.03794549, -0.00156241,  0.01644262,\n         0.00941419,  0.04181551,  0.00214124,  0.03139172,  0.00894576],\n       [ 0.05442519, -0.00588445,  0.04795203, -0.00082518, -0.00670053,\n         0.04474138, -0.00983867,  0.00324713, -0.03618562, -0.02781733],\n       [ 0.02939963, -0.02087076,  0.01900849, -0.00048406,  0.03254359,\n         0.00955461,  0.00745306, -0.01641549, -0.00286414,  0.01378625],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [ 0.01927666, -0.03023124,  0.02860137, -0.00494231,  0.01542143,\n         0.03431033,  0.00952308,  0.01740644, -0.02891522,  0.02941261],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [ 0.03671436,  0.00314417,  0.0109983 , -0.00495546,  0.04075463,\n         0.0037617 ,  0.03574388, -0.0109647 , -0.0160749 ,  0.01961437],\n       [ 0.06386209, -0.07851919, -0.00815698,  0.02147346,  0.07546461,\n        -0.00799961,  0.09047983,  0.00778581, -0.05347355,  0.01452831],\n       [ 0.01689945, -0.03179177,  0.0436387 , -0.02912539,  0.03751833,\n         0.02293188, -0.00537562, -0.00815539, -0.02665525,  0.03316449],\n       [ 0.03822878, -0.01095936,  0.01820967,  0.00592197,  0.0352524 ,\n        -0.00441891,  0.04030004, -0.01324043, -0.0123911 ,  0.01661185],\n       [ 0.03905611, -0.03626762,  0.04027533, -0.01865385, -0.00065699,\n        -0.01539162,  0.05714434, -0.00382851, -0.04040355,  0.00991061],\n       [ 0.01716395, -0.03309877,  0.01515065,  0.03909361, -0.02562316,\n        -0.00285572,  0.04892607,  0.00013886, -0.01977189, -0.03396294],\n       [ 0.00853717, -0.03805243,  0.00970995,  0.0213368 ,  0.02429332,\n        -0.01392098,  0.01894752,  0.00901922, -0.02115065,  0.00327003],\n       [ 0.01183973, -0.04745114, -0.01618965,  0.00339759, -0.01174291,\n         0.00504119,  0.01784882, -0.04434686, -0.03423206, -0.01691118],\n       [ 0.01440069, -0.01409001,  0.02721709,  0.00079619,  0.0237075 ,\n        -0.00773952,  0.04083687, -0.01151693, -0.02680119,  0.01122445],\n       [ 0.0191947 , -0.01805526,  0.00029254, -0.00261324,  0.08844699,\n        -0.03231882,  0.0200405 ,  0.00792139, -0.0696134 ,  0.04946987],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [ 0.01990443, -0.01768761,  0.00779222, -0.01193392,  0.03599756,\n         0.01173844,  0.05056381, -0.02938824, -0.01112747,  0.01877716],\n       [ 0.0319646 , -0.01231386,  0.00600453,  0.02608012,  0.03096028,\n         0.0141266 , -0.00791034, -0.02888824, -0.00096788,  0.01742369],\n       [ 0.06203877, -0.0883895 ,  0.04764955, -0.01860139,  0.08318187,\n         0.02162869,  0.10291039, -0.02690179, -0.04376375,  0.05216849],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [-0.01631242,  0.02357899, -0.01923269, -0.00248124,  0.05090763,\n        -0.01367478, -0.00650151, -0.00653741,  0.01185825,  0.03003472],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [ 0.07113041, -0.05779506,  0.03909116,  0.01664225,  0.08603613,\n         0.01298789,  0.04635942, -0.07804432,  0.03335983,  0.04454708],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [-0.04060315, -0.02760217,  0.04939116,  0.00106136, -0.04029654,\n         0.00128184,  0.02414087,  0.04006101, -0.042698  , -0.03666686],\n       [ 0.00761797,  0.00411417, -0.03647897, -0.04315722,  0.04424518,\n        -0.02224209,  0.00631681,  0.02305739, -0.00118523,  0.00490207],\n       [ 0.01021073, -0.00052577,  0.02584493,  0.00797313, -0.02019255,\n        -0.00981782, -0.01118811, -0.0027621 ,  0.00940717, -0.0156143 ],\n       [ 0.04236692, -0.04396732,  0.03935667,  0.00793953,  0.04562882,\n         0.01658781,  0.03262458, -0.03379571, -0.01384527,  0.01625991],\n       [ 0.03185312, -0.01420461,  0.0049637 , -0.03399039,  0.02408155,\n         0.049046  , -0.01253709, -0.0357851 ,  0.02716055,  0.0241413 ],\n       [ 0.06548824, -0.04096427,  0.01825981,  0.00186106,  0.06276339,\n         0.00853079,  0.05644462, -0.05234379,  0.01113156,  0.03772336],\n       [ 0.03353411, -0.03271136, -0.00850796,  0.0333568 , -0.01453083,\n        -0.03320286, -0.00683607,  0.01671835, -0.00962082, -0.00943987],\n       [ 0.04716408,  0.01261634, -0.01532903, -0.0373843 ,  0.02250929,\n        -0.03398682,  0.00061118, -0.00887315, -0.01917954,  0.00349821],\n       [ 0.04197073, -0.01281853, -0.01207545, -0.04576102, -0.00165785,\n        -0.01311673, -0.00230476, -0.0202621 , -0.03291134,  0.01878889],\n       [ 0.011799  , -0.00317613, -0.02557554,  0.02898699,  0.00273605,\n         0.03028652,  0.00683488, -0.03592701,  0.00595001, -0.0171807 ],\n       [ 0.07113041, -0.05779506,  0.03909116,  0.01664225,  0.08603613,\n         0.01298789,  0.04635942, -0.07804432,  0.03335983,  0.04454708],\n       [ 0.00559188, -0.0162238 , -0.00911005,  0.01572971,  0.01476075,\n         0.02993877,  0.02314006, -0.01739975,  0.00496858, -0.00126977],\n       [ 0.0380497 , -0.02257319,  0.00113436, -0.0176183 ,  0.04951694,\n        -0.01088668,  0.04790652, -0.00087009, -0.02819483,  0.00759832],\n       [-0.00653709, -0.00579186,  0.00996953, -0.03098773,  0.00549078,\n        -0.04145089,  0.01254475, -0.01192015, -0.03581644,  0.00414545],\n       [ 0.00357158, -0.02134069, -0.00121621, -0.00199528,  0.01479097,\n         0.00989859,  0.02569865, -0.01573609,  0.00110165,  0.00860241],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [ 0.04077195,  0.01471971, -0.0242613 ,  0.02007431, -0.02067147,\n        -0.00581208, -0.04258989, -0.03322458,  0.03644106,  0.04753356],\n       [ 0.02313958, -0.04148852,  0.02847241, -0.04356059,  0.06972939,\n         0.01322858,  0.03677382, -0.00754808, -0.03592066,  0.02578595],\n       [ 0.06537765, -0.03880406,  0.02118024,  0.0035974 ,  0.05908517,\n        -0.00031035,  0.04285981, -0.06747935,  0.02510452,  0.043793  ],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [ 0.07113041, -0.05779506,  0.03909116,  0.01664225,  0.08603613,\n         0.01298789,  0.04635942, -0.07804433,  0.03335983,  0.04454708],\n       [ 0.04563755, -0.01934028, -0.00156781, -0.02253177,  0.00084205,\n         0.01042077,  0.00727212, -0.01290583, -0.03295208, -0.00223466],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [ 0.03379446,  0.01839917,  0.01310228, -0.0504428 ,  0.02570088,\n         0.02392813,  0.0177518 ,  0.04009368, -0.03837869,  0.02042469],\n       [ 0.04034695, -0.02721812, -0.01747917, -0.01615282, -0.03564136,\n        -0.02770886, -0.00454781,  0.00420203, -0.0185065 ,  0.00172131],\n       [ 0.04034695, -0.02721812, -0.01747917, -0.01615282, -0.03564136,\n        -0.02770886, -0.00454781,  0.00420203, -0.0185065 ,  0.00172131],\n       [ 0.04474057, -0.0128208 ,  0.01749131, -0.00906393,  0.00561041,\n        -0.00680953,  0.01258418, -0.02533379, -0.02410722,  0.0014576 ],\n       [ 0.08085749, -0.05489737,  0.03428826, -0.01392128,  0.09187966,\n         0.01427231,  0.0517883 , -0.06157367, -0.05715145,  0.01669191],\n       [ 0.01933391,  0.01170437,  0.00847245, -0.01638648, -0.00725807,\n        -0.01884697, -0.00939015,  0.00417979, -0.01526869,  0.0157172 ],\n       [ 0.03718298,  0.0209762 ,  0.00594991,  0.00314582,  0.01563494,\n         0.00137906,  0.0058321 , -0.01030804, -0.01246859,  0.01346115],\n       [ 0.03179958, -0.04955486,  0.02388275, -0.05354497,  0.07020045,\n         0.01041808,  0.01323878, -0.00607392,  0.02164435,  0.04627685],\n       [ 0.07113041, -0.05779506,  0.03909116,  0.01664225,  0.08603613,\n         0.01298789,  0.04635942, -0.07804432,  0.03335983,  0.04454708],\n       [-0.01034037, -0.0246106 ,  0.00215252,  0.00163861, -0.01284114,\n        -0.00826124,  0.00902473, -0.02911858,  0.00955067, -0.03318367],\n       [-0.01034037, -0.0246106 ,  0.00215252,  0.00163861, -0.01284114,\n        -0.00826124,  0.00902473, -0.02911858,  0.00955067, -0.03318367],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [ 0.01601363, -0.01116807, -0.00528637, -0.02123079,  0.03195595,\n        -0.02608732,  0.00886789, -0.01627146, -0.03959123,  0.02219851],\n       [-0.01034037, -0.0246106 ,  0.00215252,  0.00163861, -0.01284114,\n        -0.00826124,  0.00902473, -0.02911858,  0.00955067, -0.03318367],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n       [ 0.02020101,  0.00081045,  0.01880072, -0.01665117,  0.0342534 ,\n        -0.01112656, -0.000114  , -0.01733461, -0.01468685, -0.00553192],\n       [ 0.0318405 , -0.02953796,  0.01871673,  0.00131687,  0.03026672,\n         0.01392644,  0.02156845, -0.04554027, -0.02197094,  0.02143171],\n       [ 0.02231227, -0.01177387, -0.01303811,  0.00946078,  0.02557942,\n         0.00773972,  0.00850351,  0.0087908 , -0.0219288 , -0.02109814],\n       [ 0.00171381, -0.04686017,  0.00308733,  0.0043453 , -0.00615897,\n        -0.00711915,  0.03916525,  0.00640836, -0.03105237,  0.01129271],\n       [ 0.02997982, -0.02976496,  0.00779958,  0.00257951,  0.03633864,\n         0.01430343, -0.00519789, -0.04669564, -0.00555901,  0.0170904 ],\n       [-0.03657052, -0.00012857,  0.04061024,  0.02066118, -0.00450687,\n         0.01655781,  0.0245095 ,  0.01481971,  0.01216169,  0.04099451],\n       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n         0.        ,  0.        ,  0.        ,  0.        ,  0.        ]])"
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "from collections import OrderedDict\n",
    "import numpy as np\n",
    "\n",
    "def tfidf_wtd_avg_word_vectors(words,tfidf_vector,tfidf_vocabulary,model,num_features):\n",
    "    #print(\"tfidf_vector\", tfidf_vector)\n",
    "    word_tfidfs = []\n",
    "    word_tfidf_map = OrderedDict()\n",
    "\n",
    "    for (key, proba) in tfidf_vector:\n",
    "        word_tfidfs.append(proba)\n",
    "        word_tfidf_map[tfidf_vocabulary[key]] = proba\n",
    "    \n",
    "    # print(word_tfidfs)\n",
    "    # print(word_tfidf_map)\n",
    "    \n",
    "    feature_vector=np.zeros((num_features,),dtype='float64')\n",
    "    vocabulary=set(model.wv.index2word)\n",
    "    wts=0\n",
    "    for word in words:\n",
    "        if word in vocabulary:\n",
    "            word_vector=model[word]\n",
    "            weighted_word_vector=word_tfidf_map[word]*word_vector\n",
    "            wts=wts+word_tfidf_map[word]\n",
    "            feature_vector=np.add(feature_vector,weighted_word_vector)\n",
    "    if wts:\n",
    "        feature_vector=np.divide(feature_vector,wts)\n",
    "    return feature_vector\n",
    "\n",
    "def tfidf_weighted_averaged_word_vectorizer(corpus,tfidf_vectors,tfidf_vocabulary,model,num_features):\n",
    "    docs_tfidfs=[(doc,doc_tfidf) for doc,doc_tfidf in zip(corpus,tfidf_vectors)]\n",
    "    features=[tfidf_wtd_avg_word_vectors(tokenized_sentence,tfidf,tfidf_vocabulary,model,num_features) for tokenized_sentence,tfidf in docs_tfidfs]\n",
    "    return np.array(features)\n",
    "\n",
    "# vec = Word2Vec(splitNormalWords,size=10,window=10,min_count=2,sample=1e-3)\n",
    "word2vec = Word2Vec(splitNormalWords, min_count=3, sample=1e-3, size=10, window=10)\n",
    "vector = tfidf_weighted_averaged_word_vectorizer(corpus=splitNormalWords, tfidf_vectors=tfidf_vec, tfidf_vocabulary=tfidf_vocabs,model=word2vec,num_features=10)\n",
    "vector = np.array(vector)\n",
    "# vec.build_vocab(splitNormalWords)\n",
    "# vec.train(splitNormalWords, total_examples=len(splitNormalWords), epochs=vec.epochs)\n",
    "# # vec.save(\"vec.gsm\")\n",
    "# vec\n",
    "print(len(vector))\n",
    "word2vec.save(\"word2vec.gsm\")\n",
    "vector[0:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "1479\n1105 374\n931 213\n1144\n"
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "_totalNormalSize = len(allNormalTextArray)\n",
    "_totalAdSize = len(allAdTextArray)\n",
    "actualNormalSize = _totalNormalSize\n",
    "actualAdSize = _totalAdSize\n",
    "\n",
    "y = np.concatenate((np.zeros(_totalNormalSize), np.ones(_totalAdSize)), axis=0)\n",
    "#y = (np.zeros(len(allNormalTextArray)), np.ones(len(allAdTextArray)))\n",
    "\n",
    "_i = 0\n",
    "docVector = []\n",
    "\n",
    "for v in vector:\n",
    "    mean = np.mean(v)\n",
    "\n",
    "    if mean < 0.00000001:\n",
    "        if _i >= _totalNormalSize:\n",
    "            actualAdSize -= 1\n",
    "        else:\n",
    "            actualNormalSize -= 1\n",
    "    else:\n",
    "        docVector.append(v)\n",
    "\n",
    "    _i += 1\n",
    "\n",
    "print(len(vector))\n",
    "print(_totalNormalSize, _totalAdSize)\n",
    "print(actualNormalSize, actualAdSize)\n",
    "print(len(docVector))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "0.7927927927927928\n"
    },
    {
     "output_type": "error",
     "ename": "TypeError",
     "evalue": "_predict_proba() missing 1 required positional argument: 'X'",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-45-c4f79b0ec475>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msvmModelRes\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 10\u001b[1;33m \u001b[0msvmModelRes\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m: _predict_proba() missing 1 required positional argument: 'X'"
     ]
    }
   ],
   "source": [
    "y = np.concatenate((np.zeros(_totalNormalSize), np.ones(_totalAdSize)), axis=0)\n",
    "#y = (np.zeros(len(allNormalTextArray)), np.ones(len(allAdTextArray)))\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(vector, y, test_size=0.30, random_state=100)\n",
    "\n",
    "svmModel = SVC(probability=True)  # 使用RBF核\n",
    "svmModelRes = svmModel.fit(X_train, y_train)\n",
    "print(svmModelRes.score(X_test, y_test))\n",
    "\n",
    "svmModelRes.predict_proba()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "1479\n1479\n"
    },
    {
     "output_type": "error",
     "ename": "ValueError",
     "evalue": "Negative values in data passed to MultinomialNB (input X)",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-166-9423e316c1dc>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[0mclf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mMultinomialNB\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0malpha\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfit_prior\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m \u001b[0mclf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvector\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvector\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\RT\\Anaconda3\\lib\\site-packages\\sklearn\\naive_bayes.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[0;32m    633\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    634\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_init_counters\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn_effective_classes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_features\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 635\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mY\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    636\u001b[0m         \u001b[0malpha\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_check_alpha\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    637\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_update_feature_log_prob\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0malpha\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\RT\\Anaconda3\\lib\\site-packages\\sklearn\\naive_bayes.py\u001b[0m in \u001b[0;36m_count\u001b[1;34m(self, X, Y)\u001b[0m\n\u001b[0;32m    754\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mY\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    755\u001b[0m         \u001b[1;34m\"\"\"Count and smooth feature occurrences.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 756\u001b[1;33m         \u001b[0mcheck_non_negative\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"MultinomialNB (input X)\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    757\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfeature_count_\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0msafe_sparse_dot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mY\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mT\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    758\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclass_count_\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mY\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\RT\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36mcheck_non_negative\u001b[1;34m(X, whom)\u001b[0m\n\u001b[0;32m    992\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    993\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mX_min\u001b[0m \u001b[1;33m<\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 994\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Negative values in data passed to %s\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mwhom\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    995\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    996\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Negative values in data passed to MultinomialNB (input X)"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "import pandas as pd\n",
    "\n",
    "# y = trainer.train_model()\n",
    "y = np.concatenate((np.zeros(len(allNormalTextArray)), np.ones(len(allAdTextArray))), axis=0)\n",
    "\n",
    "print(len(y))\n",
    "print(len(vector))\n",
    "\n",
    "clf = MultinomialNB(alpha=1, fit_prior=True)\n",
    "clf.fit(vector, y)\n",
    "\n",
    "result = pd.DataFrame(vector)\n",
    "keys = []\n",
    "values = []\n",
    "\n",
    "for key, value in vector:\n",
    "    keys.append(key)\n",
    "    values.append(value)\n",
    "\n",
    "df = pd.DataFrame(data={\"key\": keys, \"value\": values})\n",
    "colnames = df.sort_values(\"value\")[\"key\"].values\n",
    "result.columns = colnames\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(textMatrix, y, test_size=0.10, random_state=100)\n",
    "print(\"Test Acc\", clf.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "'0.22.1'"
     },
     "metadata": {},
     "execution_count": 167
    }
   ],
   "source": [
    "import sklearn\n",
    "sklearn.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PyCharm (JupyerNotebook)",
   "language": "python",
   "name": "pycharm-87025479"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "3.7.6-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}